{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "75 ['/data/data/fintuning/infinity-instruct/7M/train-00013-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00005-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00016-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00014-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00011-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00022-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00017-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00052-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00001-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00071-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00072-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00033-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00000-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00035-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00023-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00058-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00008-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00053-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00059-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00042-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00028-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00002-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00069-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00018-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00066-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00048-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00063-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00004-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00046-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00020-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00070-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00030-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00024-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00027-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00038-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00043-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00060-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00073-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00040-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00006-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00074-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00062-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00067-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00049-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00047-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00036-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00039-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00009-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00064-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00045-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00068-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00044-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00051-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00037-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00054-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00032-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00029-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00065-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00056-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00034-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00003-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00041-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00021-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00007-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00010-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00015-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00057-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00061-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00025-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00050-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00019-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00026-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00012-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00055-of-00075.parquet', '/data/data/fintuning/infinity-instruct/7M/train-00031-of-00075.parquet']\n"
     ]
    }
   ],
   "source": [
    "from glob import glob\n",
    "import pyarrow.parquet as pq\n",
    "import pandas as pd\n",
    "ROOT_DIR = \"/data/data/fintuning/infinity-instruct/7M\"\n",
    "all_files = glob(ROOT_DIR+\"/*\")\n",
    "print(len(all_files), all_files)\n",
    "from tqdm import tqdm\n",
    "import json\n",
    "from transformers import AutoTokenizer\n",
    "model_path = \"/data/model/llm/hf_model/steel-llm-step-1060000-ckpt\"\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 转为llama factory需要的格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/75 [00:00<?, ?it/s]Token indices sequence length is longer than the specified maximum sequence length for this model (39306 > 32768). Running this sequence through the model will result in indexing errors\n",
      "  1%|▏         | 1/75 [00:02<02:37,  2.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2410 odd_cov_counter:0 encounter:89901\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 2/75 [00:04<02:42,  2.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2400 odd_cov_counter:0 encounter:90014\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▍         | 3/75 [00:06<02:41,  2.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2488 odd_cov_counter:0 encounter:89819\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 4/75 [00:08<02:31,  2.14s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2401 odd_cov_counter:0 encounter:89917\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 5/75 [00:10<02:26,  2.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2464 odd_cov_counter:0 encounter:89885\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 6/75 [00:12<02:21,  2.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2378 odd_cov_counter:0 encounter:89873\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 7/75 [00:14<02:20,  2.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2459 odd_cov_counter:0 encounter:89898\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|█         | 8/75 [00:16<02:13,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2462 odd_cov_counter:0 encounter:89967\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▏        | 9/75 [00:18<02:10,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2431 odd_cov_counter:0 encounter:89898\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 10/75 [00:20<02:07,  1.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2413 odd_cov_counter:0 encounter:90031\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▍        | 11/75 [00:22<02:09,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2495 odd_cov_counter:0 encounter:89725\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 12/75 [00:24<02:07,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2391 odd_cov_counter:0 encounter:90071\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 13/75 [00:26<02:06,  2.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2427 odd_cov_counter:1 encounter:89777\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▊        | 14/75 [00:28<02:01,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2473 odd_cov_counter:0 encounter:89872\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 15/75 [00:30<01:59,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2539 odd_cov_counter:0 encounter:89856\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██▏       | 16/75 [00:32<01:53,  1.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2360 odd_cov_counter:0 encounter:90090\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 17/75 [00:34<01:56,  2.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2351 odd_cov_counter:0 encounter:90074\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 18/75 [00:36<01:52,  1.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2483 odd_cov_counter:0 encounter:89768\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 19/75 [00:38<01:46,  1.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2440 odd_cov_counter:0 encounter:89913\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 20/75 [00:40<01:51,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2393 odd_cov_counter:0 encounter:90051\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 21/75 [00:42<01:48,  2.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2446 odd_cov_counter:0 encounter:89772\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 22/75 [00:44<01:46,  2.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2476 odd_cov_counter:0 encounter:89846\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███       | 23/75 [00:46<01:46,  2.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2485 odd_cov_counter:0 encounter:89701\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 24/75 [00:48<01:42,  2.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2450 odd_cov_counter:0 encounter:89868\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 25/75 [00:50<01:38,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2348 odd_cov_counter:0 encounter:90090\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▍      | 26/75 [00:52<01:35,  1.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2446 odd_cov_counter:0 encounter:89981\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 27/75 [00:54<01:36,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2423 odd_cov_counter:0 encounter:89920\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 28/75 [00:56<01:30,  1.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2477 odd_cov_counter:0 encounter:90004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▊      | 29/75 [00:58<01:28,  1.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2529 odd_cov_counter:0 encounter:89966\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 30/75 [01:00<01:29,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2479 odd_cov_counter:1 encounter:89663\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████▏     | 31/75 [01:02<01:26,  1.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2456 odd_cov_counter:0 encounter:89928\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 32/75 [01:04<01:25,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2559 odd_cov_counter:0 encounter:89683\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 33/75 [01:06<01:23,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2406 odd_cov_counter:0 encounter:90021\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 34/75 [01:08<01:19,  1.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2368 odd_cov_counter:0 encounter:89985\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 35/75 [01:10<01:20,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2445 odd_cov_counter:0 encounter:90046\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 36/75 [01:12<01:16,  1.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2362 odd_cov_counter:0 encounter:89808\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▉     | 37/75 [01:13<01:14,  1.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2464 odd_cov_counter:0 encounter:89765\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████     | 38/75 [01:16<01:13,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2398 odd_cov_counter:0 encounter:89980\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 39/75 [01:18<01:11,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2415 odd_cov_counter:1 encounter:89948\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 40/75 [01:19<01:08,  1.96s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2474 odd_cov_counter:1 encounter:89937\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▍    | 41/75 [01:20<00:57,  1.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:49106 muti_turn_counter:1191 odd_cov_counter:0 encounter:44031\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 42/75 [01:22<00:57,  1.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2447 odd_cov_counter:0 encounter:89842\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 43/75 [01:24<00:56,  1.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2421 odd_cov_counter:0 encounter:89802\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▊    | 44/75 [01:26<00:56,  1.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2386 odd_cov_counter:0 encounter:90023\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 45/75 [01:28<00:54,  1.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2524 odd_cov_counter:0 encounter:89874\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████▏   | 46/75 [01:30<00:54,  1.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2436 odd_cov_counter:0 encounter:89867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 47/75 [01:32<00:52,  1.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2400 odd_cov_counter:1 encounter:90082\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▍   | 48/75 [01:34<00:52,  1.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2391 odd_cov_counter:0 encounter:90059\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 49/75 [01:36<00:51,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2472 odd_cov_counter:0 encounter:89960\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 50/75 [01:38<00:48,  1.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2431 odd_cov_counter:0 encounter:89769\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 51/75 [01:40<00:45,  1.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2309 odd_cov_counter:0 encounter:90244\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 52/75 [01:41<00:41,  1.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2427 odd_cov_counter:0 encounter:89838\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 53/75 [01:43<00:41,  1.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2498 odd_cov_counter:0 encounter:89905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 54/75 [01:45<00:40,  1.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2370 odd_cov_counter:0 encounter:89982\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 55/75 [01:47<00:38,  1.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2436 odd_cov_counter:0 encounter:89855\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▍  | 56/75 [01:49<00:37,  1.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2411 odd_cov_counter:0 encounter:90042\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 57/75 [01:51<00:36,  2.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2436 odd_cov_counter:1 encounter:89753\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 58/75 [01:53<00:33,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2396 odd_cov_counter:0 encounter:89905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▊  | 59/75 [01:55<00:30,  1.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2421 odd_cov_counter:0 encounter:89917\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 60/75 [01:57<00:29,  1.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2468 odd_cov_counter:0 encounter:89821\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████▏ | 61/75 [01:59<00:28,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2431 odd_cov_counter:0 encounter:89981\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 62/75 [02:01<00:26,  2.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2377 odd_cov_counter:0 encounter:89992\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 63/75 [02:03<00:24,  2.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2384 odd_cov_counter:0 encounter:89809\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 64/75 [02:05<00:22,  2.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2444 odd_cov_counter:0 encounter:89987\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 65/75 [02:07<00:20,  2.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2433 odd_cov_counter:0 encounter:90001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 66/75 [02:09<00:18,  2.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2375 odd_cov_counter:0 encounter:89904\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 67/75 [02:11<00:15,  1.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2393 odd_cov_counter:0 encounter:90007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 68/75 [02:13<00:13,  1.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2479 odd_cov_counter:0 encounter:89777\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 69/75 [02:15<00:11,  1.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2421 odd_cov_counter:0 encounter:90054\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 70/75 [02:17<00:09,  1.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2421 odd_cov_counter:0 encounter:89869\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▍| 71/75 [02:19<00:07,  1.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2437 odd_cov_counter:0 encounter:89954\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 72/75 [02:21<00:05,  1.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2476 odd_cov_counter:0 encounter:89754\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 73/75 [02:23<00:03,  1.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2463 odd_cov_counter:0 encounter:89824\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▊| 74/75 [02:25<00:01,  1.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2439 odd_cov_counter:0 encounter:90171\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 75/75 [02:27<00:00,  1.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all data:100000 muti_turn_counter:2473 odd_cov_counter:0 encounter:89826\n",
      "751313\n",
      "751313\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def to_alpaca_data(str_json_data):\n",
    "    return {\n",
    "                \"instruction\": str_json_data[-2][\"value\"],\n",
    "                \"input\": \"\",\n",
    "                \"output\": str_json_data[-1][\"value\"]\n",
    "            }\n",
    "\n",
    "def to_share_gpt_data(str_json_data):\n",
    "    conversations = str_json_data.tolist()\n",
    "    text = \"\"\n",
    "    for item in conversations:\n",
    "        text += item[\"value\"]\n",
    "    return {\n",
    "        \"conversations\": conversations\n",
    "    }, text\n",
    "\n",
    "def process_one_data(file_dir):\n",
    "    table = pq.read_table(file_dir)\n",
    "    df = table.to_pandas()\n",
    "    # Index(['id', 'conversations', 'label', 'langdetect', 'source'], dtype='object')\n",
    "    # print(df.columns)\n",
    "    # 可以过滤=======================\n",
    "    # need_idx = df[\"langdetect\"] == \"zh-cn\"\n",
    "    # df = df.loc[need_idx, :]\n",
    "    #===============================\n",
    "    str_json_datas = df[\"conversations\"]\n",
    "    language_datas = df[\"langdetect\"]\n",
    "    muti_turn_counter = 0\n",
    "    odd_cov_counter = 0\n",
    "    encounter = 0\n",
    "    a_file_datas = []\n",
    "    text_list = []\n",
    "    for idx, str_json_data in  enumerate(str_json_datas):\n",
    "        L = len(str_json_data)\n",
    "        # 过滤中文\n",
    "        if language_datas.iloc[idx] != \"zh-cn\":\n",
    "            encounter += 1\n",
    "            continue\n",
    "        if L > 2:\n",
    "            muti_turn_counter += 1\n",
    "        if L % 2 == 1:\n",
    "            odd_cov_counter += 1\n",
    "        # a_file_datas.append(to_alpaca_data(str_json_data)) \n",
    "        conversation, text= to_share_gpt_data(str_json_data)\n",
    "        text_list.append(text)\n",
    "        a_file_datas.append(conversation)\n",
    "    \n",
    "    input_ids = tokenizer(text_list)['input_ids']\n",
    "    token_num_list = [len(x) for x in input_ids]\n",
    "    print(f\"all data:{len(str_json_datas)} muti_turn_counter:{muti_turn_counter} odd_cov_counter:{odd_cov_counter} encounter:{encounter}\")\n",
    "    return a_file_datas,token_num_list\n",
    "\n",
    "all_datas = []\n",
    "token_nums = []\n",
    "for file_dir in tqdm(all_files):\n",
    "    datas,token_num_list = process_one_data(file_dir) \n",
    "    all_datas.extend(datas) \n",
    "    token_nums.extend(token_num_list)\n",
    "print(len(all_datas))\n",
    "print(len(token_nums))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 统计分位数，过滤过长的样本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 252.  281.  317.  368.  474.  666. 1692.]\n",
      "[247. 274. 306. 349. 423. 504. 679.]\n",
      "722236\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.percentile(token_nums,[50, 60, 70, 80, 90,95,99])\n",
    "print(np.percentile(token_nums,[50, 60, 70, 80, 90,95, 99]))\n",
    "filtered_long_data = []\n",
    "filtered_token_num = []\n",
    "for i in range(len(all_datas)):\n",
    "    if token_nums[i] < 768:\n",
    "        filtered_long_data.append(all_datas[i])\n",
    "        filtered_token_num.append(token_nums[i])\n",
    "print(np.percentile(filtered_token_num,[50, 60, 70, 80, 90,95, 99]))\n",
    "print(len(filtered_token_num))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "json_dir = \"/data/data/fintuning/llamafactory_input/infinity_chinese_data.jsonl\"\n",
    "with open(json_dir, 'w', encoding=\"utf-8\") as f:\n",
    "    for item in filtered_long_data:\n",
    "        json.dump(item, f, ensure_ascii=False)\n",
    "        f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict'>\n",
      "{'conversations': [{'from': 'human', 'value': 'A new fuel cell for notebook PCs, more compact and powerful than competing technologies, could be on the market in early 2006 at a price of around US\\\\$90, its Japanese inventors said Tuesday.\\n\\nWhat best summarizes the content of the above article?\\npick from the following. (I). World. (II). Sports. (III). Business. (IV). Science/Tech.'}, {'from': 'gpt', 'value': '(IV).'}]}\n"
     ]
    }
   ],
   "source": [
    "# 随机保存7w个数据\n",
    "import random\n",
    "random_data = random.sample(all_datas, 100000)\n",
    "print(type(all_datas[0]))\n",
    "import json\n",
    "print(all_datas[0])\n",
    "json_dir = \"/data/data/fintuning/infinity-instruct/all_data_10w.json\"\n",
    "with open(json_dir, \"w\") as file:\n",
    "    json.dump(all_datas, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open(\"/data/data/fintuning/infinity-instruct/7M/all_data_7M_v1.json\", \"r\") as file:\n",
    "    read_data = json.load(file)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
